Task 3: Experimentation

1. Comparison of Closed Form and Gradient Descent Approaches

Comparisons are made with the text features ignored. The features used here are therefore:

  1. Is root or not
  2. Number of child comments
  3. Controversiality
In [1]:
import json
import pickle

# Standard plotly imports
import plotly.graph_objs as go
import plotly.plotly as py
from plotly.offline import iplot

from pathlib import Path

# Data science imports
import pandas as pd
import numpy as np

# Cufflinks wrapper on plotly
import cufflinks
cufflinks.go_offline()

# Set the global theme for cufflinks
cufflinks.set_config_file(world_readable=True, theme='solar', offline=True)

cur_dir = !pwd
project_dir = Path(cur_dir[0]).resolve().parents[0]
features_path = project_dir / 'src' / 'features'
predictions_path = project_dir / 'reports'
In [2]:
Y_train = pickle.load(open(features_path / 'training_y.pkl', 'rb'))
predictions_train = json.load(open(predictions_path / 'predictions_train.json'))

predictions_train contains results for the following models:

In [3]:
for i, p in enumerate(predictions_train):
    print('%d. %s' % (i, p['name']))

0. ClosedForm_train
1. ClosedForm_160_train
2. ClosedForm_60_train
3. ClosedForm_no_text_train
4. GradientDescent_train
5. GradientDescent_160_train
6. GradientDescent_60_train
7. GradientDescent_no_text_train

The hyperparameters for the GD models above are:

In [4]:
predictions_train[4]['hparams']
Out[4]:
{'w_0': [0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0,
  0],
 'beta': 0.0001,
 'eta_0': 1e-05,
 'eps': 1e-06}

Comparing ClosedForm_no_text_train and GradientDescent_no_text_train:

In [5]:
cf_train_no_text = predictions_train[3]
gd_train_no_text = predictions_train[7]

cf_prediction = cf_train_no_text['y_predicted']
gd_prediction = gd_train_no_text['y_predicted']

trace1 = go.Scatter(
    x=[i for i in range(1, 10001)],
    y=[y for [y] in Y_train.tolist()],
    name = 'Target',
    line = dict(
        color='#F9A746',
        width=1),
    connectgaps=True
)
trace2 = go.Scatter(
    x=[i for i in range(1, 10001)],
    y=cf_prediction,
    name = 'Closed form',
    line = dict(
        color='#E85285',
        width=1),
    connectgaps=True
)
trace3 = go.Scatter(
    x=[i for i in range(1, 10001)],
    y=gd_prediction,
    name = 'Gradient descent',
    line = dict(
        color='#6A1B9A',
        width=1),
    connectgaps=True
)
data = [trace1, trace2, trace3]

layout = dict(title='Popularity Prediction on Training Data',
              xaxis=dict(title='Example no.'),
              yaxis=dict(title='Popularity score'),
              plot_bgcolor = '#151516')

fig = dict(data=data, layout=layout)
iplot(fig)

#df = pd.read_json(json.dumps(predictions_train))
#df

Observations:

  1. In general, closed form predicts higher popularity than gradient descent (higher peak values)
  2. In majority, the closed form solution predicts more accurately than gradient descent
  3. Both models fails to predict more extreme popularity scores i.e. high and low peaks, which implies underfitting and emphasizes the low number of features. This makes sense as we're not currently using text features
In [6]:
cf_error = (np.array([[p] for p in cf_prediction]) - Y_train)**2
gd_error = (np.array([[p] for p in gd_prediction]) - Y_train)**2


trace1 = go.Scatter(
    x=[i for i in range(1, 10001)],
    y=[e for [e] in cf_error.tolist()],
    name = 'Closed form',
    line = dict(
        color='#E85285',
        width=0.5),
    connectgaps=True
)
trace2 = go.Scatter(
    x=[i for i in range(1, 10001)],
    y=[e for [e] in gd_error.tolist()],
    name = 'Gradient descent',
    line = dict(
        color='#6A1B9A',
        width=0.5),
    connectgaps=True
)
data = [trace1, trace2]

layout = dict(title='Squared Error for Each Training Example',
              xaxis=dict(title='Example no.'),
              yaxis=dict(title='Squared error'))

fig = dict(data=data, layout=layout)
iplot(fig)

Observations:

Not much to say about this chart that hasn't been said. It was plotly practice, I guess.

To test the hypothesis that the models are underfitting without text features, we will check theirs accuracy on the validation set:

In [7]:
predictions_validate = json.load(open(predictions_path / 'predictions_validate.json'))
for i, p in enumerate(predictions_validate):
    print('%d. %s' % (i, p['name']))

0. ClosedForm_validate
1. ClosedForm_160_validate
2. ClosedForm_60_validate
3. ClosedForm_no_text_validate
4. GradientDescent_validate
5. GradientDescent_160_validate
6. GradientDescent_60_validate
7. GradientDescent_no_text_validate
In [8]:
cf_validate_no_text = predictions_validate[3]
gd_validate_no_text = predictions_validate[7]

cf_prediction = cf_validate_no_text['y_predicted']
gd_prediction = gd_validate_no_text['y_predicted']

trace1 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=[y for [y] in Y_train.tolist()],
    name = 'Target',
    line = dict(
        color='#F9A746',
        width=1),
    connectgaps=True
)
trace2 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=cf_prediction,
    name = 'Closed form',
    line = dict(
        color='#E85285',
        width=1),
    connectgaps=True
)
trace3 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=gd_prediction,
    name = 'Gradient descent',
    line = dict(
        color='#6A1B9A',
        width=1),
    connectgaps=True
)
data = [trace1, trace2, trace3]

layout = dict(title='Popularity Prediction on Validation Data',
              xaxis=dict(title='Example no.'),
              yaxis=dict(title='Popularity score'),
              plot_bgcolor = '#151516')

fig = dict(data=data, layout=layout)
iplot(fig)

Observations:

Many popularity peaks are not captured by the model, which reinforces the underfitting hypothesis

In [9]:
trace = go.Table(
    header=dict(values=['<b>Set</b>',
                        '<b>Model</b>',
                        '<b>MSE</b>',
                        '<b>Hyperparameters</b>',
                        '<b>Iterations</b>']),
    cells=dict(values=[['Training', 'Training', 'Validation', 'Validation'],
                       ['Closed Form', 'Gradient Descent', 'Closed Form', 'Gradient Descent'],
                       [cf_train_no_text['mse'],
                        gd_train_no_text['mse'],
                        cf_validate_no_text['mse'],
                        gd_validate_no_text['mse']],
                       ['', str(gd_train_no_text['hparams']),
                        '', str(gd_validate_no_text['hparams'])],
                       ['', gd_train_no_text['num_iterations'],
                        '', gd_validate_no_text['num_iterations']]
                      ]))

layout = dict(title='Summary')
data = [trace] 
iplot(dict(data=data,layout=layout))

Observations:

The summary table seems to suggest there is no underfitting, as the models performed better on the validation set than on the training set. However, it is possible it is due to random sampling and noise.

On stability:

The CF's weight vector is computed from a fixed formula - it will therefore give the same prediction for the same inputs.
On the other hand, GD's prediction depends on the hyperparameters set at training stage - different GD models will thus yield different predictions even for the same inputs.

This means GD is more unstable than CF.

Conclusions:

  1. Overall, the closed form model performed better than gradient descent (better MSE for both sets)
  2. Gradient descent is more unstable than closed form
  3. Closed form has better runtime during training (because of its straightfoward computation), but there is still risk of obtaining singular matrices

2. No Text Features vs 160 Words vs 60 Words

Comparisons will be made using the closed form model.

In [10]:
cf_train_160 = predictions_train[1]
cf_train_60 = predictions_train[2]
cf_validate_160 = predictions_validate[1]
cf_validate_60 = predictions_validate[2]

trace = go.Table(
    header=dict(values=['',
                        '<b>No Text</b>',
                        '<b>Top 60</b>',
                        '<b>Top 160</b>']),
    cells=dict(values=[['<b>Training</b>','<b>Validation</b>'],
                       [cf_train_no_text['mse'], cf_validate_no_text['mse']],
                       [cf_train_60['mse'], cf_validate_60['mse']],
                       [cf_train_160['mse'], cf_validate_160['mse']]
                      ]))

layout = dict(title='Closed Form MSE')
data = [trace] 
iplot(dict(data=data,layout=layout))
In [16]:
cf_y_no_text = cf_validate_no_text['y_predicted']
cf_y_60 = cf_validate_60['y_predicted']
cf_y_160 = cf_validate_160['y_predicted']

trace1 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=[y for [y] in Y_train.tolist()],
    name = '(white) Target',
    line = dict(
        color='white',
        width=1),
    connectgaps=True
)
trace2 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=cf_y_no_text,
    name = 'No Text',
    line = dict(
        color='#F9A746',
        width=1),
    connectgaps=True
)
trace3 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=cf_y_60,
    name = 'Top 60',
    line = dict(
        color='#E85285',
        width=1),
    connectgaps=True
)
trace4 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=cf_y_160,
    name = 'Top 160',
    line = dict(
        color='#6A1B9A',
        width=1),
    connectgaps=True
)
data = [trace1, trace2, trace3, trace4]

layout = dict(title='Closed Form Predictions on Validation Data',
              xaxis=dict(title='Example no.'),
              yaxis=dict(title='Popularity score'),
              plot_bgcolor = '#151516'
             )

fig = dict(data=data, layout=layout)
iplot(fig)

Observations:

Judging from the MSEs, top-60 is an incremental improvement over no text, and top-160 is an incremental improvement over top-60.
Even though top-160's validation MSE is the only one higher than its training's, it's a small difference (1.0477763217987115 and 1.0686390774956736).
The fact that top-160's validation MSE is lower than no text's is demonstrative that top-60 gives the best predictions out of the three.

Conclusions:

Adding the top 160 words features overall improves our model.

3. New Features: Comment Length and Porter Stemming

In [12]:
cf_train = predictions_train[0]
cf_validate = predictions_validate[0]

trace = go.Table(
    header=dict(values=['',
                        '<b>Top 160</b>',
                        '<b>Full</b> (top 160, length and stemming)']),
    cells=dict(values=[['<b>Training</b>','<b>Validation</b>'],
                       [cf_train_160['mse'], cf_validate_160['mse']],
                       [cf_train['mse'], cf_validate['mse']]
                      ]))

layout = dict(title='Closed Form MSE')
data = [trace] 
iplot(dict(data=data,layout=layout))

4. Running on Test Set

In [13]:
predictions_test = json.load(open(predictions_path / 'predictions_test.json'))
for i, p in enumerate(predictions_test):
    print('%d. %s' % (i, p['name']))

0. ClosedForm_test
1. ClosedForm_160_test
2. ClosedForm_60_test
3. ClosedForm_no_text_test
4. GradientDescent_test
5. GradientDescent_160_test
6. GradientDescent_60_test
7. GradientDescent_no_text_test
In [14]:
cf_test_160 = predictions_test[1]
cf_test = predictions_test[0]

trace = go.Table(
    header=dict(values=['',
                        '<b>Top 160</b>',
                        '<b>Full</b> (top 160, length and stemming)']),
    cells=dict(values=[['<b>Training</b>','<b>Validation</b>', '<b>Test</b>'],
                       [cf_train_160['mse'], cf_validate_160['mse'], cf_test_160['mse']],
                       [cf_train['mse'], cf_validate['mse'], cf_test['mse']]
                      ]))

layout = dict(title='Closed Form MSE')
data = [trace] 
iplot(dict(data=data,layout=layout))
In [17]:
cf_y_160 = cf_test_160['y_predicted']
cf_y = cf_test['y_predicted']

trace1 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=[y for [y] in Y_train.tolist()],
    name = 'Target',
    line = dict(
        color='#F9A746',
        width=1),
    connectgaps=True
)
trace2 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=cf_y,
    name = 'Full',
    line = dict(
        color='#E85285',
        width=1),
    connectgaps=True
)
trace3 = go.Scatter(
    x=[i for i in range(1, 1001)],
    y=cf_y_160,
    name = 'Top 160',
    line = dict(
        color='#6A1B9A',
        width=1),
    connectgaps=True
)
data = [trace1, trace2, trace3]

layout = dict(title='Closed Form Predictions on Test Data',
              xaxis=dict(title='Example no.'),
              yaxis=dict(title='Popularity score'),
              plot_bgcolor = '#151516'
             )

fig = dict(data=data, layout=layout)
iplot(fig)

Observations:

The full model (2 new features) only outperforms the top-160 model with the training set, and is worse in both with the validation and test set

Conclusions:

Our best model overfits and I'm sad.